package com.eric.common.net;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.commons.logging.LogFactory;
import org.apache.log4j.PropertyConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

/*根据返回码进行判断，如若网站限制爬虫，
则模拟浏览器进行下载*/
public class SimplePageDownloader {

	private String strUrl;

	Logger logger;

	private int timeout = 5000;

	public static void main(String[] args) throws IOException {
		String s = "http://www.1688.com/robots.txt";
		SimplePageDownloader spd = new SimplePageDownloader(s);
		System.out.println(spd.download());
	}

	public SimplePageDownloader(String strUrl) {
		this.strUrl = strUrl;
		logger = LoggerFactory.getLogger(SimplePageDownloader.class);
		PropertyConfigurator.configure("log4j.properties");
	}

	public String download(String strUrl) {
		this.strUrl = strUrl;
		return download();
	}

	public SimplePageDownloader setTimeOut(int timeout) {
		this.timeout = timeout;
		return this;
	}
	
	public String download () {
		String html = null;
		
		if (crawlable(strUrl)) {
			logger.info("Download by spider");
			html = downloadBySpider(strUrl);
		}else{
			logger.info("Download by simulating browser action");
			html = downloadByBrowse(strUrl);
		}
		
		return html;
	}

//	public String download() {
//		String html = null;
//		Response response = null;
//		int status = 0;
//		try {
//			response = Jsoup.connect(strUrl).timeout(timeout).followRedirects(true).execute();
//			status = response.statusCode();
//		} catch (org.jsoup.HttpStatusException e) {
//			status = e.getStatusCode();
//		} catch (IOException e) {
//			e.printStackTrace();
//		}
//		if (status != 200) {
//			logger.info("Download by simulating browser action");
//			return downloadByBrowse(strUrl);
//		}
//		try {
//			logger.info("Download by crawler");
//			html = response.parse().toString();
//		} catch (IOException e) {
//			e.printStackTrace();
//		}
//		return html;
//	}

	public int getResponseCode(String url) {
		int status = 0;
		try {
			URL URL = new URL(url);
			HttpURLConnection conn = (HttpURLConnection) URL.openConnection();
			status = conn.getResponseCode();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return status;
	}

	public boolean crawlable(String url) {
		int status = getResponseCode(url);
		if (status == 200 || status == 301 || status == 302)
			return true;
		return false;
	}

	// 模拟浏览器行为下载页面以及js
	public String downloadByBrowse(String strUrl) {
		URL url;
		HtmlPage page = null;
		java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(java.util.logging.Level.OFF);
		LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
				"org.apache.commons.logging.impl.NoOpLog");
		try {
			url = new URL(strUrl);
			final WebClient client = new WebClient();
			client.getOptions().setJavaScriptEnabled(true);
			client.getOptions().setRedirectEnabled(true);
			client.getOptions().setThrowExceptionOnScriptError(false);
			client.getOptions().setThrowExceptionOnFailingStatusCode(false);
			client.getOptions().setTimeout(timeout);
			page = client.getPage(url);
			client.close();
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (FailingHttpStatusCodeException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return page.asXml();
	}
	
	public String downloadBySpider(String strUrl) {
		URL url;
		HttpURLConnection conn;
		StringBuffer sb = new StringBuffer();
		
		try {
			url = new URL(strUrl);
			conn = (HttpURLConnection) url.openConnection();
			
			boolean redirect = false;
			
			// normally, 3xx is redirect
			int status = conn.getResponseCode();
			if (status != HttpURLConnection.HTTP_OK) {
				if (status == HttpURLConnection.HTTP_MOVED_TEMP
					|| status == HttpURLConnection.HTTP_MOVED_PERM
						|| status == HttpURLConnection.HTTP_SEE_OTHER)
				redirect = true;
			}

			if (redirect) {

				// get redirect url from "location" header field
				String newStrUrl = conn.getHeaderField("Location");

				// get the cookie if need, for login
				String cookies = conn.getHeaderField("Set-Cookie");

				// open the new connnection again
				conn = (HttpURLConnection) new URL(newStrUrl).openConnection();
				conn.setRequestProperty("Cookie", cookies);
				conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8");
				conn.addRequestProperty("User-Agent", "Mozilla");
				conn.addRequestProperty("Referer", "baidu.com");
			}
			
			BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
			
			String line;
			while ((line = br.readLine()) != null) {
				sb.append("\n");
				sb.append(line);
			}
			br.close();
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
			
		return sb.toString();
	}

}
